# Install 7z
!apt-get install -y p7zip
# Extract datasets
!7z x -y MAXZEGA.7z
!7z x -y IZ0CVY.7z
!7z x -y SV2HWM.7z
#!7z x -y IU8QAI.7z
### Merge all datasets in a huge one
## Currently not used
##!if [ ! -f MERGED.csv ]; then cp MAXZEGA.csv MERGED.csv; tail -n +2 IZ0CVY.csv >> MERGED.csv; tail -n +2 SV2HWM.csv >> MERGED.csv; fi
Reading package lists... Done Building dependency tree Reading state information... Done p7zip is already the newest version (16.02+dfsg-6). 0 upgraded, 0 newly installed, 0 to remove and 39 not upgraded. 7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21 p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI) Scanning the drive for archives: 0M Scan 1 file, 95857562 bytes (92 MiB) Extracting archive: MAXZEGA.7z -- Path = MAXZEGA.7z Type = 7z Physical Size = 95857562 Headers Size = 130 Method = BZip2 Solid = - Blocks = 1 0% 2% - MAXZEGA.csv 4% - MAXZEGA.csv 7% - MAXZEGA.csv 10% - MAXZEGA.csv 12% - MAXZEGA.csv 15% - MAXZEGA.csv 17% - MAXZEGA.csv 20% - MAXZEGA.csv 23% - MAXZEGA.csv 25% - MAXZEGA.csv 28% - MAXZEGA.csv 29% - MAXZEGA.csv 30% - MAXZEGA.csv 33% - MAXZEGA.csv 35% - MAXZEGA.csv 38% - MAXZEGA.csv 41% - MAXZEGA.csv 43% - MAXZEGA.csv 46% - MAXZEGA.csv 49% - MAXZEGA.csv 51% - MAXZEGA.csv 54% - MAXZEGA.csv 57% - MAXZEGA.csv 59% - MAXZEGA.csv 61% - MAXZEGA.csv 63% - MAXZEGA.csv 66% - MAXZEGA.csv 69% - MAXZEGA.csv 71% - MAXZEGA.csv 73% - MAXZEGA.csv 76% - MAXZEGA.csv 79% - MAXZEGA.csv 82% - MAXZEGA.csv 84% - MAXZEGA.csv 87% - MAXZEGA.csv 89% - MAXZEGA.csv 92% - MAXZEGA.csv 95% - MAXZEGA.csv 98% - MAXZEGA.csv 99% - MAXZEGA.csv Everything is Ok Size: 418660575 Compressed: 95857562 7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21 p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI) Scanning the drive for archives: 0M Scan 1 file, 73478268 bytes (71 MiB) Extracting archive: IZ0CVY.7z -- Path = IZ0CVY.7z Type = 7z Physical Size = 73478268 Headers Size = 130 Method = LZMA2:24 Solid = - Blocks = 1 0% 2% - IZ0CVY.csv 5% - IZ0CVY.csv 7% - IZ0CVY.csv 10% - IZ0CVY.csv 13% - IZ0CVY.csv 15% - IZ0CVY.csv 18% - IZ0CVY.csv 21% - IZ0CVY.csv 23% - IZ0CVY.csv 26% - IZ0CVY.csv 28% - IZ0CVY.csv 31% - IZ0CVY.csv 34% - IZ0CVY.csv 36% - IZ0CVY.csv 39% - IZ0CVY.csv 42% - IZ0CVY.csv 44% - IZ0CVY.csv 47% - IZ0CVY.csv 49% - IZ0CVY.csv 52% - IZ0CVY.csv 55% - IZ0CVY.csv 57% - IZ0CVY.csv 60% - IZ0CVY.csv 63% - IZ0CVY.csv 65% - IZ0CVY.csv 68% - IZ0CVY.csv 70% - IZ0CVY.csv 72% - IZ0CVY.csv 75% - IZ0CVY.csv 78% - IZ0CVY.csv 80% - IZ0CVY.csv 83% - IZ0CVY.csv 86% - IZ0CVY.csv 88% - IZ0CVY.csv 91% - IZ0CVY.csv 93% - IZ0CVY.csv 94% - IZ0CVY.csv 96% - IZ0CVY.csv 98% - IZ0CVY.csv Everything is Ok Size: 477379756 Compressed: 73478268 7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21 p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI) Scanning the drive for archives: 0M Scan 1 file, 22450501 bytes (22 MiB) Extracting archive: SV2HWM.7z -- Path = SV2HWM.7z Type = 7z Physical Size = 22450501 Headers Size = 130 Method = BZip2 Solid = - Blocks = 1 0% 9% - SV2HWM.csv 17% - SV2HWM.csv 28% - SV2HWM.csv 37% - SV2HWM.csv 47% - SV2HWM.csv 54% - SV2HWM.csv 64% - SV2HWM.csv 72% - SV2HWM.csv 78% - SV2HWM.csv 86% - SV2HWM.csv 94% - SV2HWM.csv 98% - SV2HWM.csv Everything is Ok Size: 103557518 Compressed: 22450501
# Set the station callsign
CALLSIGN = "MAXZEGA"
# Import libraries
import numpy as np
import seaborn as sn
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib.pyplot import figure
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits import mplot3d
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
df = pd.read_csv('/content/' + CALLSIGN + '.csv', sep=';')
# Ignore strong signals to avoid biases
df = df.query("(snr > 0) | (snr < 70)")
# Remove unused (test) columns
df = df.drop('tx_pow', axis='columns')
df = df.drop('rec_pow', axis='columns')
df.head(10)
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 41.66827 | 12.45356 | 2275.4 | 1.44 | 176.71 | 70.3 | 8.4 | 6.0 | 7.2 | 128.1 | -273.0 | -1.0 | -1.0 |
| 1 | 41.66743 | 12.45469 | 2353.4 | 1.50 | 176.64 | 70.4 | 9.3 | 7.7 | 12.8 | 141.9 | -273.0 | -1.0 | -1.0 |
| 2 | 41.66717 | 12.45491 | 2371.4 | 1.51 | 176.62 | 70.5 | 9.9 | 7.1 | 9.3 | 147.1 | -273.0 | -1.0 | -1.0 |
| 3 | 41.66710 | 12.45497 | 2377.6 | 1.52 | 176.62 | 70.5 | 9.9 | 6.4 | 8.6 | 139.6 | -273.0 | -1.0 | -1.0 |
| 4 | 41.66704 | 12.45505 | 2383.1 | 1.52 | 176.61 | 70.5 | 9.0 | 5.0 | 9.7 | 130.6 | -273.0 | -1.0 | -1.0 |
| 5 | 41.66698 | 12.45515 | 2388.7 | 1.53 | 176.61 | 70.5 | 8.9 | 6.7 | 11.8 | 126.3 | -273.0 | -1.0 | -1.0 |
| 6 | 41.66643 | 12.45595 | 2433.0 | 1.56 | 176.56 | 70.6 | 10.9 | 6.0 | 9.9 | 123.7 | -273.0 | -1.0 | -1.0 |
| 7 | 41.66638 | 12.45605 | 2438.5 | 1.57 | 176.55 | 70.6 | 10.8 | 5.5 | 10.6 | 120.8 | -273.0 | -1.0 | -1.0 |
| 8 | 41.66627 | 12.45627 | 2449.6 | 1.57 | 176.54 | 70.6 | 10.2 | 7.0 | 12.4 | 129.0 | -273.0 | -1.0 | -1.0 |
| 9 | 41.66599 | 12.45661 | 2466.5 | 1.59 | 176.51 | 70.6 | 10.1 | 6.2 | 15.2 | 148.4 | -273.0 | -1.0 | -1.0 |
df.describe()
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 |
| mean | 4.210061e+01 | 1.256262e+01 | 1.813788e+04 | 8.880707e+00 | 1.847788e+02 | 1.346245e+02 | 1.458357e+01 | 4.609758e-02 | 1.756067e+01 | 1.368936e+02 | -5.032971e+01 | 1.182207e+01 | 1.184640e+01 |
| std | 1.063755e+00 | 1.137396e+00 | 9.217938e+03 | 7.389827e+00 | 7.763717e+01 | 7.584136e+01 | 3.975513e+00 | 1.065985e+01 | 1.214361e+01 | 7.726429e+01 | 3.111094e+01 | 2.086772e+01 | 4.597079e+01 |
| min | 3.916920e+01 | 7.721560e+00 | 3.898000e+02 | 1.600000e-01 | 0.000000e+00 | 1.250000e+01 | -2.000000e-01 | -1.337000e+02 | 0.000000e+00 | 0.000000e+00 | -2.730000e+02 | -1.000000e+00 | -1.000000e+00 |
| 25% | 4.149547e+01 | 1.222952e+01 | 1.054080e+04 | 3.770000e+00 | 1.396800e+02 | 7.640000e+01 | 1.150000e+01 | -3.000000e-01 | 8.900000e+00 | 8.250000e+01 | -5.920000e+01 | 1.200000e+00 | -1.000000e+00 |
| 50% | 4.171277e+01 | 1.260628e+01 | 1.786600e+04 | 6.740000e+00 | 1.677800e+02 | 1.092000e+02 | 1.440000e+01 | 4.500000e+00 | 1.470000e+01 | 1.128000e+02 | -5.480000e+01 | 1.800000e+00 | -1.000000e+00 |
| 75% | 4.212912e+01 | 1.306472e+01 | 2.547050e+04 | 1.198000e+01 | 1.914300e+02 | 1.782000e+02 | 1.730000e+01 | 5.500000e+00 | 2.290000e+01 | 1.784000e+02 | -4.280000e+01 | 1.060000e+01 | -1.000000e+00 |
| max | 4.650279e+01 | 1.802704e+01 | 3.800030e+04 | 8.311000e+01 | 3.600000e+02 | 5.112000e+02 | 3.360000e+01 | 8.010000e+01 | 1.087000e+02 | 3.600000e+02 | 3.140000e+01 | 1.000000e+02 | 8.941000e+02 |
ddf = df
fig = plt.figure(figsize=(16, 16))
fig.set_dpi(90)
# AZIMUTH vs SNR
ax1 = plt.subplot(1,3,1, projection = 'polar')
ax1.set_theta_zero_location("N")
ax1.scatter(-np.deg2rad(ddf.azimuth), ddf.snr, s=0.2, cmap='jet')
plt.ylim(0, 45)
# ax1.set_xlabel('azimuth', fontsize = 20, labelpad = 20)
ax1.set_title('Azimuth vs SNR')
# ELEVATION vs SNR
ax2 = fig.add_subplot(1,3,2, projection='polar', xlim=(0, 90))
ax2.set_theta_zero_location("N")
ax2.set_thetamin(-90) # set the limits
ax2.set_thetamax(90)
ax2.set_theta_offset(np.deg2rad(0)) # point the origin towards the top
plt.ylim(0, 35)
# ax2.set_thetagrids(range(-90, 120, 30)) # set the gridlines
ax2.scatter(np.deg2rad(ddf.elevation), ddf.snr, s=0.2, cmap='jet')
# plt.ylim(0, 45)
ax2.set_title('Elevation vs SNR')
# AZIMUTH vs ELEVATION with SNR
ax3 = fig.add_subplot(1,3,3, projection='polar')
ax3.set_theta_zero_location("N")
# Create chart
plot = ax3.scatter(-np.deg2rad(ddf['azimuth']), ddf['elevation'], s=0.2, c=ddf['snr'], cmap='jet')
ax3.set_title('Azimuth vs Elevation (with SNR)')
# Create legend
cbaxes = inset_axes(ax3, width="1%", height="30%", loc=1)
plt.colorbar(plot, cax=cbaxes, orientation='vertical', ticks=np.linspace(ddf['snr'].min(), ddf['snr'].max(), 6))
# Show chart
plt.show()
# DISTANCE vs SNR
fig = plt.figure(figsize=(18, 5))
fig.set_dpi(85)
ax3 = fig.add_subplot(1,1,1)
plt.grid()
plt.scatter(ddf.distance, ddf.snr, s=0.1)
plt.xlim(0, 500)
plt.ylim(0, 40)
# ax3.set_xlabel('distance', fontsize = 20, labelpad = 20)
ax3.set_title('Distance vs SNR')
ax3.set_xticks(np.linspace(0,500,11))
plt.show()
# # produce a legend with the unique colors from the scatter
# legend1 = ax.legend(*scatter.legend_elements(),
# loc="lower left", title="Classes")
# ax.add_artist(legend1)
# ax3 = fig.add_subplot(111, projection = '3d')
# # ax3.set_theta_zero_location("N")
# plot = ax3.scatter(ddf.distance, ddf.snr, s=0.2, cmap='jet')
# ax3.set_xlabel('distance', fontsize = 20, labelpad = 20)
# ax3.set_title('Distance vs snr')
# Copy the dataframe, and drop unused columns
ddf = df
ddf = ddf.drop('latitude', axis='columns')
ddf = ddf.drop('longitude', axis='columns')
#ddf = ddf.drop('tx_pow', axis='columns')
#ddf = ddf.drop('rec_pow', axis='columns')
fig = plt.figure(figsize=(16, 16))
fig.set_dpi(75)
# Create the correlation matrix
corr_matrix = ddf.corr(method='spearman')
sn.heatmap(corr_matrix, annot=True)
# Show the correlation matrix
plt.show()
# Copy the dataframe
ddf = df
ddf.describe()
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 | 5.024064e+06 |
| mean | 4.210061e+01 | 1.256262e+01 | 1.813788e+04 | 8.880707e+00 | 1.847788e+02 | 1.346245e+02 | 1.458357e+01 | 4.609758e-02 | 1.756067e+01 | 1.368936e+02 | -5.032971e+01 | 1.182207e+01 | 1.184640e+01 |
| std | 1.063755e+00 | 1.137396e+00 | 9.217938e+03 | 7.389827e+00 | 7.763717e+01 | 7.584136e+01 | 3.975513e+00 | 1.065985e+01 | 1.214361e+01 | 7.726429e+01 | 3.111094e+01 | 2.086772e+01 | 4.597079e+01 |
| min | 3.916920e+01 | 7.721560e+00 | 3.898000e+02 | 1.600000e-01 | 0.000000e+00 | 1.250000e+01 | -2.000000e-01 | -1.337000e+02 | 0.000000e+00 | 0.000000e+00 | -2.730000e+02 | -1.000000e+00 | -1.000000e+00 |
| 25% | 4.149547e+01 | 1.222952e+01 | 1.054080e+04 | 3.770000e+00 | 1.396800e+02 | 7.640000e+01 | 1.150000e+01 | -3.000000e-01 | 8.900000e+00 | 8.250000e+01 | -5.920000e+01 | 1.200000e+00 | -1.000000e+00 |
| 50% | 4.171277e+01 | 1.260628e+01 | 1.786600e+04 | 6.740000e+00 | 1.677800e+02 | 1.092000e+02 | 1.440000e+01 | 4.500000e+00 | 1.470000e+01 | 1.128000e+02 | -5.480000e+01 | 1.800000e+00 | -1.000000e+00 |
| 75% | 4.212912e+01 | 1.306472e+01 | 2.547050e+04 | 1.198000e+01 | 1.914300e+02 | 1.782000e+02 | 1.730000e+01 | 5.500000e+00 | 2.290000e+01 | 1.784000e+02 | -4.280000e+01 | 1.060000e+01 | -1.000000e+00 |
| max | 4.650279e+01 | 1.802704e+01 | 3.800030e+04 | 8.311000e+01 | 3.600000e+02 | 5.112000e+02 | 3.360000e+01 | 8.010000e+01 | 1.087000e+02 | 3.600000e+02 | 3.140000e+01 | 1.000000e+02 | 8.941000e+02 |
# Filter results to examine the lower atmosphere
ddf = df.query('altitude <= 14000')
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig("/content/sns_pairplot_" + CALLSIGN + "_14000-.png")
# Filter results to examine the higher atmosphere
ddf = df.query('altitude >= 20000')
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig("/content/sns_pairplot_" + CALLSIGN + "_20000+.png")
# Filter results to examine the lower atmosphere
ddf = df
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig("/content/sns_pairplot_" + CALLSIGN + "_fullrange.png")